import json
import logging
import puzautils.table.puza_row as pr
import puzautils.table.puza_stream_table as pst
import puzautils.twitter_api.twitter_crawler as tc

seed_table_name = 'users_1_profiles.remain.db'
save_table_name = 'users_1_profiles.remain.crawled'
log_filename = 'users_1_profiles.remain.log'

flag = 0

def ReadPool():
    global seed_table_name
    global flag
    InputTable = pst.PuzaStreamTable()
    InputTable.InputFile(seed_table_name)
    pool = set()
    while True:
        raw_obj = InputTable.StreamInRow()
        if raw_obj != None:
            r = pr.PuzaRow.Load(raw_obj)
            if r['id'] != None:
                pool.add(r['id'])
                flag = 1
            else:
                pool.add(r['screenname'])
                flag = 0
        else:
            break
    return pool       

if __name__ == '__main__':    
    logging.basicConfig(filename= log_filename, filemode='w', level=logging.INFO, format='%(asctime)s::%(levelname)s::[%(module)s::%(funcName)s]::%(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
    logging.info('Starting to read all users')
    user_pool = ReadPool()
    logging.info('Finish to read all users')
    logging.info('Total number of users:' + str(len(user_pool)))
    c = tc.TwitterCrawler('day')
    c.Auth()
    OutputTable = pst.PuzaStreamTable()
    OutputTable.OutputFile(save_table_name)
    while True:
        names = []
        if len(user_pool) >= 100:
            for k in range(100):
                names.append(str(user_pool.pop()))
        elif len(user_pool) > 0:            
            for i in user_pool:
                names.append(str(i))
            user_pool = set()
        else:
            break
        if flag == 0:
            all_profiles = c.GetBulkScreenNames(names)
        else:
            all_profiles = c.GetBulkIDs(names)
        all_profiles = unicode(all_profiles,'utf-8')
        try:
            json_obj = json.loads(all_profiles)
            for obj in json_obj:
                r = pr.PuzaRow(obj)
                OutputTable.StreamOutRow(r, lformat =0)
        except:
            continue
    OutputTable.StreamClose()
    logging.info('Finished.')